install.packages("RSelenium")
install.packages("rvest")
install.packages("tidyverse")

require(RSelenium)
require(rvest)
require(tidyverse)

rD <- rsDriver(browser="firefox", port=4545L, verbose=F)
remDr <- rD[["client"]]

webpage <- 
  "https://www.meteonetwork.it/rete/dati.php?region=&type=daily"

remDr$navigate(webpage)

dates <- 
  seq(from = as.Date("2020-01-01"), 
      to = as.Date("2020-06-30"),
      by = 'day')
dates <- format(dates, "%d-%m-%Y")

weather_data.df <- data.frame()

for (this_date in dates) {
  
  print(this_date)
  
  # 1. Date
  date_input <- 
    remDr$findElement("xpath", '//*[@id="datepick"]')
  date_input$clearElement()
  date_input$sendKeysToElement(list(this_date))
  date_submit <- 
    remDr$findElement("xpath", '/html/body/div[3]/div[3]/form/input[2]')
  date_submit$clickElement()
  
  # 2. Selector
  option <- 
    remDr$findElement(using = 'xpath', "/html/body/div[3]/div[2]/div[1]/form/select/option[4]")
  option$clickElement()
  
  # 3. Parse table
  Sys.sleep(5)
  html_source <- 
    remDr$getPageSource()[[1]]
  
  parsed <- 
    xml2::read_html(html_source)
  xml2::xml_find_all(parsed, './/td/span[@class="red"]') %>% xml2::xml_remove()
  table <- 
    parsed %>% xml2::xml_find_first('.//table') %>% 
    html_table() %>%
    dplyr::mutate(date = this_date)
  
  hrefs <- 
    xml2::read_html(html_source) %>%
    html_nodes(xpath = "/html/body/div[3]/table/tbody/tr/td[1]/span/a") %>%
    html_attr("href")
  
  stations <- 
    gsub("http://my.meteonetwork.it/station/|/", "", hrefs)
  
  table$code <- 
    stations
  
  weather_data.df <- 
    rbind(weather_data.df, table)
  
}

save(weather_data.df, file = "~/Downloads/weather_data.df.RData")


remDr <- rD$client
remDr$close()
rm(rD)
gc()



